# coding:utf-8
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.utils import shuffle 
'''
Created on 2018年2月1日
https://www.cnblogs.com/irenelin/p/7400388.html
@author: jiangpeizhao
'''

pd.set_option('display.max_rows',None)
PATH = r'D:\\Workspace\\HousePrices\\totoro\\hp\\data\\'
# df = pd.read_csv(f'{PATH}train.csv')
df = pd.read_csv(PATH + 'train.csv')
df_train = pd.read_csv(PATH + 'train.csv')
df_test = pd.read_csv(PATH + 'test.csv')

# 训练集中SalePrice的分布
fig = plt.figure()
fig.set_figheight(6)
fig.set_figwidth(12)
ax1 = fig.add_subplot(1, 2, 1)
ax2 = fig.add_subplot(1, 2, 2)
ax1.hist(df_train.SalePrice)
ax2.hist(np.log1p(df_train.SalePrice))
# plt.show()

# 根据训练集来看一下特征的分布
features = pd.concat([df_train, df_test],keys=['train','test'])
# 成数值特征和类别特征分别查看
numeric_feats = features.dtypes[features.dtypes!="object"].index
categorical_feats = features.dtypes[features.dtypes=="object"].index

for c in categorical_feats:
    df_train[c] = df_train[c].astype('category')
    if df_train[c].isnull().any():
        df_train[c] = df_train[c].cat.add_categories(["Missing"])
        df_train[c] = df_train[c].fillna("Missing")
def boxplot(x,y,**kwargs):
    sns.boxplot(x=x,y=y)
f = pd.melt(df_train,id_vars=['SalePrice'],value_vars=categorical_feats)
g = sns.FacetGrid(f,col='variable',col_wrap=3,sharex=False,sharey=False,size=5)
g = g.map(boxplot,"value","SalePrice")
# plt.show()

def jointplot(x,y,**kwargs):
    try:
        sns.regplot(x=x,y=y)
    except Exception:
        print(x.value_counts())
numeric_feats = numeric_feats.drop("SalePrice")
f = pd.melt(df_train, id_vars=['SalePrice'],value_vars=numeric_feats)
g = sns.FacetGrid(f,col='variable',col_wrap=3,sharex=False,sharey=False,size=5)
g = g.map(jointplot,"value","SalePrice")
# plt.show()

nomial_feats=['MSSubClass','OverallQual','OverallCond','YearBuilt','YearRemodAdd','BsmtFullBath','FullBath','HalfBath',
             'BedroomAbvGr','TotRmsAbvGrd','GarageYrBlt','GarageCars','MoSold','YrSold']
for c in nomial_feats:
    df_train[c] = df_train[c].astype('category')
    if df_train[c].isnull().any():
        df_train[c] = df_train[c].cat.add_categories(["Missing"])
        df_train[c] = df_train[c].fillna("Missing")
f = pd.melt(df_train,id_vars=['SalePrice'],value_vars=nomial_feats)
g = sns.FacetGrid(f,col='variable',col_wrap=3,sharex=False,sharey=False,size=5)
g = g.map(boxplot,"value","SalePrice")
# plt.show()

df_train.drop(df_train[(df_train['GrLivArea']>4000)&(df_train.SalePrice<300000)].index,inplace=True)

# 最后再来看一下变量之间的相关关系。
plt.subplots(figsize=(12,10))
corrmat = df_train.corr()
g = sns.heatmap(df_train.corr())
# plt.show()

# 数据集中缺失的情况
print(features.isnull().sum()[features.isnull().sum()>0])

# 1.missing_col_NA:缺失即为没有的用“NA”来填充,数值类型按情况补充。
missing_col_NA = ["Alley","MasVnrType","FireplaceQu","GarageType","PoolQC","Fence",
                  "MiscFeature","GarageQual","GarageCond","GarageFinish"]
missing_col_0 = ["MasVnrArea","GarageCars","GarageArea"]
for col in missing_col_NA:
    features[col].fillna("NA",inplace=True)
    
for col in missing_col_0:
    features[col].fillna(0,inplace=True)
    
# 2.missing_col_mode:有的缺失数值可以用平均值，众数来补充。
missing_col_mode = ["MSZoning","KitchenQual","Functional","SaleType","Electrical","Exterior1st","Exterior2nd","Utilities"]
for col in missing_col_mode:
    features[col].fillna(features[col].mode()[0],inplace=True)
features['LotFrontage'] = features.groupby("Neighborhood")["LotFrontage"].transform(lambda x:x.fillna(x.median()))

# 3. 缺失值中有Bsmt的都是描述地下室的，所以可以一起来处理。
#由于缺失值的个数是不一样的，所以只有BsmtCond和BsmtQual同时缺失，才将其认为是没有地下室的。
NoBmstIndex = (pd.isnull(features["BsmtCond"])==True)&(pd.isnull(features["BsmtQual"])==True)

features.loc[NoBmstIndex,"BsmtCond"] =features.loc[NoBmstIndex,"BsmtCond"].fillna("NA")
features.loc[NoBmstIndex,"BsmtQual"] =features.loc[NoBmstIndex,"BsmtQual"].fillna("NA")
features.loc[NoBmstIndex,"BsmtExposure"] =features.loc[NoBmstIndex,"BsmtExposure"].fillna("NA")

#其余的用众数来填充
features.BsmtCond.fillna(features.BsmtCond.mode()[0],inplace=True)
features.BsmtQual.fillna(features.BsmtQual.mode()[0],inplace=True)
features.BsmtExposure.fillna(features.BsmtExposure.mode()[0],inplace=True)

features.BsmtFinSF1.fillna(0,inplace=True)
features.BsmtFinSF2.fillna(0,inplace=True)
features.BsmtFinType1.fillna("NA",inplace=True)
features.BsmtFinType2.fillna("NA",inplace=True)

#要将没有地下室和地下室未完成的面积区分开，所以如果是未完成的，将其面积设为中位数
features.loc[features["BsmtFinType1"]=="Unf","BsmtFinSF1"]=features.BsmtFinSF1.median()
features.loc[features["BsmtFinType2"]=="Unf","BsmtFinSF2"]=features.BsmtFinSF1.median()

features.BsmtUnfSF.fillna(0,inplace=True)
features.TotalBsmtSF.fillna(0,inplace=True)

features.BsmtFullBath.fillna(0, inplace=True)
features.BsmtHalfBath.fillna(0, inplace=True)
print(features)

# 4.另外还有如Exterior1st 和 Exterior2nd 这样的属性，表示房子外墙的材料类型，如果有两种，第二个才会有值
features.loc[features["Exterior1st"]==features["Exterior2nd"],"Exterior2nd"]="None"
features.loc[features["Condition1"]==features["Condition2"],"Condition2"]="None"

# 5.Utilities 的取值单一，没有太大意义，故直接将其删除。
print(features.Utilities.value_counts())

# 准备训练数据    
# 1.对特征进行转换，有些数值特征的分布偏度比较大，可以通过对数变化使其变成正态分布，在一个kernel里面看到有boxcox变化也可以处理这种情况。
from scipy.special import boxcox1p
numeric_feats = features.dtypes[features.dtypes != "object"].index
skewed_feats = features[numeric_feats].apply(lambda x: x.dropna().skew()) #compute skewness
skewed_feats = skewed_feats[skewed_feats > 0.75]
skewed_feats = skewed_feats.index

#features[skewed_feats] = np.log1p(features[skewed_feats])
lam=0.15
features[skewed_feats] = boxcox1p(features[skewed_feats],lam)

# 2. 添加新的特征
Unf=['2.5Unf','1.5Unf']
features["HouseStypeFinish"]=features["HouseStyle"].map(lambda x:0 if x in Unf else 1)
# 3.删除不用的特征
features.drop(['GarageYrBlt','Id','Street','SaleType','FullBath',
               '1stFlrSF','GarageArea'],axis=1,inplace=True)

# 4. 将数值特征转为类别特征
features.YrSold = features.YrSold.astype(str)
features.MoSold = features.MoSold.astype(str)
features.MSSubClass = features.MSSubClass.astype(str)
features.HalfBath = features.HalfBath.astype(str)
features.BedroomAbvGr = features.BedroomAbvGr.astype(str)
features.GarageCars = features.GarageCars.astype(str)

# 5.将类别特征转为数值特征
features = features.replace({"Alley" : {"NA":0,"Grvl" : 1, "Pave" : 2},
                "BsmtCond" : {"NA":-1,"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
                "BsmtExposure" : {"NA":0,"No" : 1, "Mn" : 1, "Av": 2, "Gd" : 3},
                "BsmtQual" : {"NA" : -1,"No" : 0, "Po" : 1, "Fa" : 2, "TA": 3, "Gd" : 4, "Ex" : 5},
                "ExterCond" : {"NA" : -1,"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
                "ExterQual" : {"NA" : -1,"Po" : 1, "Fa" : 2, "TA": 3, "Gd": 4, "Ex" : 5},
                "FireplaceQu" : {"NA" : -1,"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
                "Functional" : {"NA" : -1,"Sal" : 1, "Sev" : 2, "Maj2" : 3, "Maj1" : 4, "Mod": 5, 
                                       "Min2" : 6, "Min1" : 7, "Typ" : 8},
                "GarageCond" : {"NA" : -1,"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
                "GarageQual" : {"NA" : -1,"No" : 0, "Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
                "HeatingQC" : {"NA" : -1,"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
                "KitchenQual" : {"NA" : -1,"Po" : 1, "Fa" : 2, "TA" : 3, "Gd" : 4, "Ex" : 5},
                "LandSlope" : {"NA" : -1,"Sev" : 1, "Mod" : 2, "Gtl" : 3},
                "LotShape" : {"NA" : -1,"IR3" : 1, "IR2" : 2, "IR1" : 3, "Reg" : 4},
                "PoolQC" : {"NA" : -1,"No" : 0, "Fa" : 1, "TA" : 2, "Gd" : 3, "Ex" : 4},
                "Street" : {"NA" : -1,"Grvl" : 1, "Pave" : 2}
                            })

# 6.将类别特征进行onehot-encode的，这样才能使用sklearn里面的模型训练。
category_feats = features.dtypes[features.dtypes == "object"].index

for col in category_feats:
    for_dummy = features.pop(col)
    extra_data = pd.get_dummies(for_dummy,prefix=col)
    #print(col,":",extra_data.shape)
    features = pd.concat([features, extra_data],axis=1)
    
# 7.归一化
from sklearn.preprocessing import RobustScaler
N = RobustScaler()
for elem in features:
    print(elem)
scale_features = N.fit_transform(features)

# 训练集和测试集
# train_features = scale_features[:df_train.shape[0]]
# test_features = scale_features[df_train.shape[0]:]
# train_labels = y.values.copy()
# 
# train_features,train_labels = shuffle(train_features,train_labels,random_state=5)
